<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>一文彻底搞懂Kafka | 全栈进阶那些事</title><meta name="keywords" content="Kafka"><meta name="author" content="liuyuantao"><meta name="copyright" content="liuyuantao"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="一文彻底搞懂Kafka">
<meta property="og:type" content="article">
<meta property="og:title" content="一文彻底搞懂Kafka">
<meta property="og:url" content="https://liuyuantao.gitee.io/2021/10/12/245dd8355f34.html">
<meta property="og:site_name" content="全栈进阶那些事">
<meta property="og:description" content="一文彻底搞懂Kafka">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://img-blog.csdnimg.cn/20200620092054385.png">
<meta property="article:published_time" content="2021-10-12T07:45:59.000Z">
<meta property="article:modified_time" content="2021-11-13T01:06:19.871Z">
<meta property="article:author" content="liuyuantao">
<meta property="article:tag" content="中间件">
<meta property="article:tag" content="Kafka">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://img-blog.csdnimg.cn/20200620092054385.png"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://liuyuantao.gitee.io/2021/10/12/245dd8355f34"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?d43383137144f76a37f1074576bc674e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":1,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"简"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":200},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: true
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":500,"languages":{"author":"作者: liuyuantao","link":"链接: ","source":"来源: 全栈进阶那些事","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#2d3035","position":"top-right"},
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '一文彻底搞懂Kafka',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2021-11-13 09:06:19'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const fontSizeVal = saveToLocal.get('global-font-size')
    if (fontSizeVal !== undefined) {
      document.documentElement.style.setProperty('--global-font-size', fontSizeVal + 'px')
    }
    
    const detectApple = () => {
      if (GLOBAL_CONFIG_SITE.isHome && /iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.0"><link rel="alternate" href="/atom.xml" title="全栈进阶那些事" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/zhiyao_avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="not-top-img" id="page-header"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">全栈进阶那些事</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav></header><main class="layout" id="content-inner"><div id="post"><div id="post-info"><h1 class="post-title">一文彻底搞懂Kafka</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="fa-fw post-meta-icon far fa-calendar-alt"></i><span class="post-meta-label">发表于</span><time datetime="2021-10-12T07:45:59.000Z" title="undefined 2021-10-12 15:45:59">2021-10-12</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/">中间件</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">字数总计:</span><span class="word-count">5.6k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">阅读时长:</span><span>23分钟</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="一文彻底搞懂Kafka"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div><article class="post-content" id="article-container"><blockquote>
<p>本文是基于CentOS 7.9系统环境，进行Kafka的学习和使用</p>
</blockquote>
<h1 id="一、Kafka的简介"><a href="#一、Kafka的简介" class="headerlink" title="一、Kafka的简介"></a>一、Kafka的简介</h1><h2 id="1-1-Kafka基本概念"><a href="#1-1-Kafka基本概念" class="headerlink" title="1.1 Kafka基本概念"></a>1.1 Kafka基本概念</h2><p><strong>(1) 什么是Kafka</strong></p>
<p>Kafka是一个分布式的基于发布/订阅模式的消息队列，主要应用于大数据实时处理领域<br><strong>(2) 消息队列</strong></p>
<ol>
<li>点对点模式的消息队列</li>
</ol>
<p>对一个消息而言，只会有一个消费者可以消费，消费者主动拉取消息，消息收到后，会将消息删除</p>
<ol start="2">
<li>基于发布/订阅模式的消息队列</li>
</ol>
<p>发布到topic的消息会被所有订阅者消费，消费者消费完消息后不会删除</p>
<ol start="3">
<li>消息队列主动推送</li>
</ol>
<p>适用于消费者处理能力高，生产者生产消息较少的场景，整个系统的消息处理速度由消息队列的推送速度决定，多个消费者之间处理消息的能力不同，如果消息队列按照一定的速度推送消息，可能会造成某些消费者处理消息处理不过来，某些消费者处理速度很快一直处于空闲</p>
<ol start="4">
<li>消费者主动拉取</li>
</ol>
<p>适用于生产者生产消息的速度高于消费者，整个系统的消息处理速度由消费者的拉取速度决定，如果消息队列中消息较少，消费者会一直维护着一个长轮询去访问消息队列是否有消息</p>
<p><strong>(3) Kafka的作用</strong></p>
<ul>
<li><strong>解耦和异步</strong></li>
</ul>
<p>将强依赖的两个数据上下游系统，通过消息队列进行解耦，可以实现上下游的异步通信</p>
<ul>
<li><strong>削峰</strong></li>
</ul>
<p>缓解数据上下游两个系统的流速差</p>
<h2 id="1-2-Kafka基础架构"><a href="#1-2-Kafka基础架构" class="headerlink" title="1.2 Kafka基础架构"></a>1.2 Kafka基础架构</h2><ol>
<li>为了方便横向扩展，并提高吞吐量，一个topic分为多个partition</li>
<li>配合分区的设计，提出消费者组的概念，组内每个消费者并行消费</li>
<li>为提高可用性，为每个partition增加若干副本，类似于NameNode HA</li>
</ol>
<p><img src="https://img-blog.csdnimg.cn/20200620092054385.png"></p>
<ul>
<li><strong>Producer</strong></li>
</ul>
<p>消息生产者，连接broker-list，就是向kafka broker发消息的客户端</p>
<ul>
<li><strong>Consumer</strong></li>
</ul>
<p>消息消费者，连接bootstrp-server，向kafka broker取消息的客户端</p>
<ul>
<li><strong>Consumer Group （CG）</strong></li>
</ul>
<p>消费者组，由多个consumer组成。消费者组内每个消费者负责消费不同分区的数据，一个分区只能由一个消费者消费；消费者组之间互不影响。所有的消费者都属于某个消费者组，即消费者组是逻辑上的一个订阅者</p>
<ul>
<li><strong>Cluster</strong></li>
</ul>
<p>kafka集群，连接zookeeper，包含多个broker</p>
<ul>
<li><strong>Broker</strong></li>
</ul>
<p>一台kafka服务器就是一个broker。一个集群由多个broker组成。一个broker可以容纳多个topic</p>
<ul>
<li><strong>Topic</strong></li>
</ul>
<p>可以理解为一个队列，生产者和消费者面向的都是一个topic</p>
<ul>
<li><strong>分区为了并发，备份为了容灾</strong></li>
<li><strong>Partition</strong></li>
</ul>
<p>为了实现扩展性，一个非常大的topic可以分布到多个broker（即服务器）上，一个topic可以分为多个partition，每个partition是一个有序的队列</p>
<ul>
<li><strong>Replica</strong></li>
</ul>
<p>副本，为保证集群中的某个节点发生故障时， 该节点上的partition数据不丢失，且kafka仍然能够继续工作，kafka提供了副本机制，一个topic的每个分区都有若干个副本，一个leader和若干个follower</p>
<ul>
<li><strong>leader</strong></li>
</ul>
<p>每个分区多个副本的“主”，生产者发送数据的对象，以及消费者消费数据的对象都是leader</p>
<ul>
<li><strong>follower</strong></li>
</ul>
<p>每个分区多个副本中的“从”，实时与leader保持数据的同步。leader发生故障时，某个follower会成为新的follower</p>
<ul>
<li><strong>zookeeper</strong></li>
</ul>
<p>存储topic信息、分区信息、副本信息、ISR信息、消费者的偏移量（0.9版本之前，之后存在kafka集群中d __consumer_offset主题中，为了提高效率，减少与zookeeper的通信时间）</p>
<h2 id="1-3-Kafka工作流程"><a href="#1-3-Kafka工作流程" class="headerlink" title="1.3 Kafka工作流程"></a>1.3 Kafka工作流程</h2><ol>
<li>kafka中消息是以topic进行分类的，生产者生成消息，消费者消费消息，都是面向topic的</li>
<li>topic是逻辑上的概念，而partition是物理上的概念，每个partition对应于一组log文件，该log文件中存储的就是producer生产的数据。</li>
<li>Producer生产的数据会被不断追加到该log文件末端，且每条数据都有自己的offset。</li>
<li>消费者组中的每个消费者，都会实时记录自己消费到了哪个offset，以便出错恢复时，从上次的位置继续消费。</li>
</ol>
<h2 id="1-4-Kafka文件存储机制"><a href="#1-4-Kafka文件存储机制" class="headerlink" title="1.4 Kafka文件存储机制"></a>1.4 Kafka文件存储机制</h2><p><img src="https://img-blog.csdnimg.cn/20200623152607642.png"></p>
<ol>
<li>由于生产者生产的消息会不断追加到log文件末尾，为防止log文件过大导致数据定位效率低下，Kafka采取了分片和索引机制，将每个partition分为多个segment</li>
<li>每个segment对应两个文件——“.index”文件和“.log”文件。这些文件位于一个文件夹下，该文件夹的命名规则为：topic名称+分区序号。例如，first这个topic有三个分区，则其对应的文件夹为first-0,first-1,first-2</li>
<li>index和log文件以当前segment的第一条消息的offset命名</li>
<li>index文件存储大量的索引信息，log文件存储大量的数据，索引文件中的元数据指向对应数据文件中message的物理偏移地址</li>
</ol>
<p><img src="https://img-blog.csdnimg.cn/20201225091515558.png"></p>
<h2 id="1-5-Kafka生产者"><a href="#1-5-Kafka生产者" class="headerlink" title="1.5 Kafka生产者"></a>1.5 Kafka生产者</h2><h3 id="1-5-1-分区策略"><a href="#1-5-1-分区策略" class="headerlink" title="1.5.1 分区策略"></a>1.5.1 分区策略</h3><p><strong>(1) 分区的原因</strong></p>
<ul>
<li>方便在集群中扩展</li>
</ul>
<p>每个Partition可以通过调整以适应它所在的机器，而一个topic又可以有多个Partition组成，因此整个集群就可以适应任意大小的数据了</p>
<ul>
<li>可以提高并发</li>
</ul>
<p>因为可以以Partition为单位读写了</p>
<p><strong>(2) 分区的原则</strong></p>
<p>我们需要将producer发送的数据封装成一个ProducerRecord对象。</p>
<ul>
<li>指明 partition 的情况下，直接将指明的值直接作为 partiton 值</li>
<li>没有指明 partition 值但有 key 的情况下，将 key 的 hash 值与 topic 的 partition 数进行取余得到 partition 值</li>
<li>既没有 partition 值又没有 key 值的情况下，第一次调用时随机生成一个整数（后面每次调用在这个整数上自增），将这个值与 topic 可用的 partition 总数取余得到 partition 值，也就是常说的 round-robin 算法</li>
</ul>
<h3 id="1-5-2-数据可靠性保证"><a href="#1-5-2-数据可靠性保证" class="headerlink" title="1.5.2 数据可靠性保证"></a>1.5.2 数据可靠性保证</h3><p>为保证producer发送的数据，能可靠的发送到指定的topic，topic的每个partition收到producer发送的数据后，都需要向producer发送ack（acknowledgement确认收到），如果producer收到ack，就会进行下一轮的发送，否则重新发送数据。</p>
<p><img src="https://img-blog.csdnimg.cn/20200623163726511.png"><br><strong>(1) 副本数据同步策略</strong><br>方策略  |  优点 |   缺点<br>|—|—|—|<br>半数以上完成同步，就发送ack  |  延迟低  |  选举新的leader时，容忍n台节点的故障，需要2n+1个副本<br>全部完成同步，就发送ack  |  选举新的leader时，容忍n台节点的故障，需要n+1个副本 |   延迟高</p>
<p>Kafka选择了第二种方案，原因如下：</p>
<ul>
<li>同样为了容忍n台节点的故障，第一种方案需要2n+1个副本，而第二种方案只需要n+1个副本，而Kafka的每个分区都有大量的数据，第一种方案会造成大量数据的冗余。</li>
<li>虽然第二种方案的网络延迟会比较高，但网络延迟对Kafka的影响较小。</li>
</ul>
<p><strong>(2) ISR</strong></p>
<p>采用第二种方案之后，设想以下情景：leader收到数据，所有follower都开始同步数据，但有一个follower，因为某种故障，迟迟不能与leader进行同步，那leader就要一直等下去，直到它完成同步，才能发送ack。这个问题怎么解决呢？</p>
<p>Leader维护了一个动态的in-sync replica set (ISR)，意为和leader保持同步的follower集合。当ISR中的follower完成数据的同步之后，leader就会给producer发送ack。如果follower长时间未向leader同步数据，则该follower将被踢出ISR，该时间阈值由replica.lag.time.max.ms参数设定。Leader发生故障之后，就会从ISR中选举新的leader。<br><strong>(3) ack应答机制</strong></p>
<p>对于某些不太重要的数据，对数据的可靠性要求不是很高，能够容忍数据的少量丢失，所以没必要等ISR中的follower全部接收成功。<br>所以Kafka为用户提供了三种可靠性级别，用户根据对可靠性和延迟的要求进行权衡，选择以下的配置。</p>
<p>acks参数配置</p>
<ul>
<li><strong>0</strong></li>
</ul>
<p>producer不等待broker的ack，这一操作提供了一个最低的延迟，broker一接收到还没有写入磁盘就已经返回，当broker故障时有可能丢失数据；</p>
<ul>
<li><strong>1</strong></li>
</ul>
<p>producer等待broker的ack，partition的leader落盘成功后返回ack，如果在follower同步成功之前leader故障，那么将会丢失数据；</p>
<ul>
<li><strong>-1（all）</strong></li>
</ul>
<p>producer等待broker的ack，partition的leader和follower全部落盘成功后才返回ack。但是如果在follower同步完成后，broker发送ack之前，leader发生故障，那么会造成数据重复。</p>
<p><strong>（4）消费数据一致性保障</strong></p>
<ul>
<li><strong>LEO</strong></li>
</ul>
<p>指的是每个副本最大的offset</p>
<ul>
<li><strong>HW</strong></li>
</ul>
<p>指的是消费者能见到的最大的offset，ISR队列中最小的LEO<br><img src="https://img-blog.csdnimg.cn/20201225095954613.png"></p>
<p><strong>（5）故障处理细节</strong></p>
<ul>
<li><strong>follower故障</strong></li>
</ul>
<p>follower发生故障后会被临时踢出ISR，待该follower恢复后，follower会读取本地磁盘记录的上次的HW，并将log文件高于HW的部分截取掉，从HW开始向leader进行同步。等该follower的LEO大于等于该Partition的HW，即follower追上leader之后，就可以重新加入ISR了。</p>
<ul>
<li><strong>leader故障</strong></li>
</ul>
<p>leader发生故障之后，会从ISR中选出一个新的leader，之后，为保证多个副本之间的数据一致性，其余的follower会先将各自的log文件高于HW的部分截掉，然后从新的leader同步数据。<br>    注意：这只能保证副本之间的数据一致性，并不能保证数据不丢失或者不重复。</p>
<h3 id="1-5-3-Exactly-Once语义"><a href="#1-5-3-Exactly-Once语义" class="headerlink" title="1.5.3 Exactly Once语义"></a>1.5.3 Exactly Once语义</h3><p>将服务器的ACK级别设置为-1，可以保证Producer到Server之间不会丢失数据，即At Least Once语义。相对的，将服务器ACK级别设置为0，可以保证生产者每条消息只会被发送一次，即At Most Once语义。</p>
<p>At Least Once可以保证数据不丢失，但是不能保证数据不重复；相对的，At most Once可以保证数据不重复，但是不能保证数据不丢失。但是，对于一些非常重要的信息，比如说交易数据，下游数据消费者要求数据既不重复也不丢失，即Exactly Once语义。在0.11版本以前的Kafka，对此是无能为力的，只能保证数据不丢失，再在下游消费者对数据做全局去重。对于多个下游应用的情况，每个都需要单独做全局去重，这就对性能造成了很大影响。</p>
<p>0.11版本的Kafka，引入了一项重大特性：幂等性。所谓的幂等性就是指Producer不论向Server发送多少次重复数据，Server端都只会持久化一条。 幂等性结合At Least Once语义，就构成了Kafka的Exactly Once语义。即：</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">At Least Once + 幂等性 = Exactly Once</span><br></pre></td></tr></table></figure>

<p>要启用幂等性，只需要将Producer的参数中enable.idempotence设置为true即可。Kafka的幂等性实现其实就是将原来下游需要做的去重放在了数据上游。开启幂等性的Producer在初始化的时候会被分配一个PID，发往同一Partition的消息会附带Sequence Number。而Broker端会对&lt;PID, Partition, SeqNumber&gt;做缓存，当具有相同主键的消息提交时，Broker只会持久化一条。</p>
<p>但是PID重启就会变化，同时不同的Partition也具有不同主键，所以幂等性无法保证跨分区跨会话的Exactly Once。</p>
<h2 id="1-6-Kafka消费者"><a href="#1-6-Kafka消费者" class="headerlink" title="1.6 Kafka消费者"></a>1.6 Kafka消费者</h2><h3 id="1-6-1-消费方式"><a href="#1-6-1-消费方式" class="headerlink" title="1.6.1 消费方式"></a>1.6.1 消费方式</h3><p>consumer采用pull（拉）模式从broker中读取数据。</p>
<p>push（推）模式很难适应消费速率不同的消费者，因为消息发送速率是由broker决定的。它的目标是尽可能以最快速度传递消息，但是这样很容易造成consumer来不及处理消息，典型的表现就是拒绝服务以及网络拥塞。而pull模式则可以根据consumer的消费能力以适当的速率消费消息。</p>
<p>pull模式不足之处是，如果kafka没有数据，消费者可能会陷入循环中，一直返回空数据。针对这一点，Kafka的消费者在消费数据时会传入一个时长参数timeout，如果当前没有数据可供消费，consumer会等待一段时间之后再返回，这段时长即为timeout。</p>
<h3 id="1-6-2-分区分配策略"><a href="#1-6-2-分区分配策略" class="headerlink" title="1.6.2 分区分配策略"></a>1.6.2 分区分配策略</h3><p>一个consumer group中有多个consumer，一个 topic有多个partition，所以必然会涉及到partition的分配问题，即确定那个partition由哪个consumer来消费。</p>
<p>Kafka有两种分配策略，一是roundrobin，一是range。</p>
<h3 id="1-6-3-offset的维护"><a href="#1-6-3-offset的维护" class="headerlink" title="1.6.3 offset的维护"></a>1.6.3 offset的维护</h3><p>由于consumer在消费过程中可能会出现断电宕机等故障，consumer恢复后，需要从故障前的位置的继续消费，所以consumer需要实时记录自己消费到了哪个offset，以便故障恢复后继续消费。</p>
<p>Kafka 0.9版本之前，consumer默认将offset保存在Zookeeper中，从0.9版本开始，consumer默认将offset保存在Kafka一个内置的topic中，该topic为__consumer_offsets。</p>
<h1 id="二、Kafka的安装和使用"><a href="#二、Kafka的安装和使用" class="headerlink" title="二、Kafka的安装和使用"></a>二、Kafka的安装和使用</h1><p>2.0 Kafka群起脚本</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="meta">#! /bin/bash</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">case</span> <span class="variable">$1</span> <span class="keyword">in</span></span><br><span class="line"><span class="string">&quot;start&quot;</span>)&#123;</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> hadoop101 hadoop102 hadoop103</span><br><span class="line">        <span class="keyword">do</span></span><br><span class="line">                <span class="built_in">echo</span> <span class="string">&quot;==========<span class="variable">$i</span>==========&quot;</span></span><br><span class="line">                ssh <span class="variable">$i</span> <span class="string">&quot;source /etc/profile; /opt/module/kafka/bin/kafka-server-start.sh -daemon /opt/module/kafka/config/server.properties&quot;</span></span><br><span class="line">        <span class="keyword">done</span></span><br><span class="line">&#125;;;</span><br><span class="line"><span class="string">&quot;stop&quot;</span>)&#123;</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> hadoop101 hadoop102 hadoop103</span><br><span class="line">        <span class="keyword">do</span></span><br><span class="line">                <span class="built_in">echo</span> <span class="string">&quot;==========<span class="variable">$i</span>==========&quot;</span></span><br><span class="line">                ssh <span class="variable">$i</span> <span class="string">&quot;source /etc/profile; /opt/module/kafka/bin/kafka-server-stop.sh&quot;</span></span><br><span class="line">        <span class="keyword">done</span></span><br><span class="line">&#125;;;</span><br><span class="line">*)&#123;</span><br><span class="line">        <span class="built_in">echo</span> <span class="string">&quot;Error: Could not find the option <span class="variable">$1</span> &quot;</span></span><br><span class="line">&#125;;;</span><br><span class="line"><span class="keyword">esac</span></span><br></pre></td></tr></table></figure>

<h2 id="2-1-Kafka安装"><a href="#2-1-Kafka安装" class="headerlink" title="2.1 Kafka安装"></a>2.1 Kafka安装</h2><figure class="highlight plaintext"><table><tr><td class="code"><pre><span class="line">tar -xzvf kafka_2.11-0.11.0.0.tgz -C /opt/module</span><br><span class="line">cd /opt/module</span><br><span class="line">mv kafka_2.11-0.11.0.0/ kafka</span><br></pre></td></tr></table></figure>

<h2 id="2-2-修改配置文件"><a href="#2-2-修改配置文件" class="headerlink" title="2.2 修改配置文件"></a>2.2 修改配置文件</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi /opt/module/kafka/config/server.properties</span><br><span class="line"><span class="comment"># 修改如下内容</span></span><br><span class="line"><span class="comment"># The id of the broker. This must be set to a unique integer for each broker.</span></span><br><span class="line">broker.id=1</span><br><span class="line"></span><br><span class="line"><span class="comment"># Switch to enable topic deletion or not, default value is false</span></span><br><span class="line">delete.topic.enable=<span class="literal">true</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># A comma seperated list of directories under which to store log files</span></span><br><span class="line">log.dirs=/opt/module/kafka/data1,/opt/module/kafka/data2</span><br><span class="line"></span><br><span class="line"><span class="comment"># root directory for all kafka znodes.</span></span><br><span class="line"><span class="comment"># 设置kafka信息在zookeeper中的存储节点路径</span></span><br><span class="line">zookeeper.connect=hadoop101:2181,hadoop102:2181,hadoop103:2181/kafka</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="2-3-配置环境变量"><a href="#2-3-配置环境变量" class="headerlink" title="2.3 配置环境变量"></a>2.3 配置环境变量</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /etc/profile</span><br><span class="line"><span class="comment"># 添加如下内容</span></span><br><span class="line"><span class="comment">#KAFKA_HOME</span></span><br><span class="line"><span class="built_in">export</span> KAFKA_HOME=/opt/module/kafka</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$PATH</span>:<span class="variable">$KAFKA_HOME</span>/bin</span><br></pre></td></tr></table></figure>

<p>保存退出，并重新打开一个终端，配置文件则会生效</p>
<h2 id="2-4-启动kafka"><a href="#2-4-启动kafka" class="headerlink" title="2.4 启动kafka"></a>2.4 启动kafka</h2><p>首先保证已经启动了zookeeper集群</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/kafka</span><br><span class="line">bin/kafka-server-start.sh -daemon config/server.properties</span><br></pre></td></tr></table></figure>

<h2 id="2-5-停止kafka"><a href="#2-5-停止kafka" class="headerlink" title="2.5 停止kafka"></a>2.5 停止kafka</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/kafka</span><br><span class="line">bin/kafka-server-stop.sh</span><br></pre></td></tr></table></figure>

<h2 id="2-5-kafka启动日志文件"><a href="#2-5-kafka启动日志文件" class="headerlink" title="2.5 kafka启动日志文件"></a>2.5 kafka启动日志文件</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/kafka/config</span><br><span class="line">vi server.properties</span><br></pre></td></tr></table></figure>

<h2 id="2-6-查看当前服务器中的所有topic"><a href="#2-6-查看当前服务器中的所有topic" class="headerlink" title="2.6 查看当前服务器中的所有topic"></a>2.6 查看当前服务器中的所有topic</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-topics.sh --zookeeper hadoop101:2181 --list</span><br></pre></td></tr></table></figure>

<h2 id="2-6-创建topic"><a href="#2-6-创建topic" class="headerlink" title="2.6 创建topic"></a>2.6 创建topic</h2><p>分区数可以大于集群节点数，但是副本数不可以大过集群副本数</p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-topics.sh --zookeeper hadoop101:2181 --create --replication-factor 3 --partitions 1 --topic first</span><br></pre></td></tr></table></figure>

<h2 id="2-7-查看topic详情"><a href="#2-7-查看topic详情" class="headerlink" title="2.7 查看topic详情"></a>2.7 查看topic详情</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-topics.sh --zookeeper hadoop101:2181 --describe --topic first</span><br></pre></td></tr></table></figure>

<h2 id="2-8-修改topic分区数量"><a href="#2-8-修改topic分区数量" class="headerlink" title="2.8 修改topic分区数量"></a>2.8 修改topic分区数量</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-topics.sh --zookeeper hadoop101:2181 --alter --topic first --partitions 6</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<h2 id="2-9-删除topic"><a href="#2-9-删除topic" class="headerlink" title="2.9 删除topic"></a>2.9 删除topic</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-topics.sh --zookeeper hadoop101:2181 --delete --topic first</span><br></pre></td></tr></table></figure>

<h2 id="2-10-发送消息"><a href="#2-10-发送消息" class="headerlink" title="2.10 发送消息"></a>2.10 发送消息</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-console-producer.sh --broker-list hadoop101:9092 --topic first</span><br></pre></td></tr></table></figure>

<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">hello world</span><br><span class="line">lytdev</span><br></pre></td></tr></table></figure>

<h2 id="2-11-从头消费消息"><a href="#2-11-从头消费消息" class="headerlink" title="2.11 从头消费消息"></a>2.11 从头消费消息</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-console-consumer.sh --bootstrap-server hadoop101:9092 --from-beginning --topic first</span><br></pre></td></tr></table></figure>

<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># kafka 0.9版本之前借助于zookeeper存储offset信息，所以要使用</span></span><br><span class="line"><span class="comment"># bin/kafka-console-consumer.sh --zookeeper hadoop101:2181 --from-beginning --topic first</span></span><br></pre></td></tr></table></figure>

<h2 id="2-12-实时消费消息"><a href="#2-12-实时消费消息" class="headerlink" title="2.12 实时消费消息"></a>2.12 实时消费消息</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-console-consumer.sh --bootstrap-server hadoop101:9092 --topic first</span><br></pre></td></tr></table></figure>

<h2 id="2-13-从上次故障恢复消费消息"><a href="#2-13-从上次故障恢复消费消息" class="headerlink" title="2.13 从上次故障恢复消费消息"></a>2.13 从上次故障恢复消费消息</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-console-consumer.sh --bootstrap-server hadoop101:9092  --topic first --consumer-property group.id=consoleGroup</span><br></pre></td></tr></table></figure>

<h1 id="三、Kafka的API"><a href="#三、Kafka的API" class="headerlink" title="三、Kafka的API"></a>三、Kafka的API</h1><h2 id="3-1-Producer-API"><a href="#3-1-Producer-API" class="headerlink" title="3.1 Producer API"></a>3.1 Producer API</h2><h3 id="3-1-1-消息发送流程"><a href="#3-1-1-消息发送流程" class="headerlink" title="3.1.1 消息发送流程"></a>3.1.1 消息发送流程</h3><p>Kafka的Producer发送消息采用的是异步发送的方式。在消息发送的过程中，涉及到了两个线程——main线程和Sender线程，以及一个线程共享变量——RecordAccumulator。main线程将消息发送给RecordAccumulator，Sender线程不断从RecordAccumulator中拉取消息发送到Kafka broker。</p>
<p><strong>相关参数</strong></p>
<ul>
<li><strong>batch.size</strong></li>
</ul>
<p>只有数据积累到batch.size之后，sender才会发送数据</p>
<ul>
<li><strong>linger.time</strong></li>
</ul>
<p>如果数据迟迟未达到batch.size，sender等待linger.time之后就会发送数据</p>
<h3 id="3-1-2-异步发送API"><a href="#3-1-2-异步发送API" class="headerlink" title="3.1.2 异步发送API"></a>3.1.2 异步发送API</h3><p><strong>（1）不带回调函数的发送消息</strong></p>
<figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.lytdevproducer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.producer.KafkaProducer;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.producer.Producer;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.producer.ProducerRecord;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.Properties;</span><br><span class="line"><span class="keyword">import</span> java.util.concurrent.Future;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">ProducerWithOutCallback</span> </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</span><br><span class="line">        Properties props = <span class="keyword">new</span> Properties();</span><br><span class="line">        props.put(<span class="string">&quot;bootstrap.servers&quot;</span>, <span class="string">&quot;hadoop101:9092&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;acks&quot;</span>, <span class="string">&quot;all&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;retries&quot;</span>, <span class="number">1</span>);</span><br><span class="line">        props.put(<span class="string">&quot;batch.size&quot;</span>, <span class="number">16384</span>);</span><br><span class="line">        props.put(<span class="string">&quot;linger.ms&quot;</span>, <span class="number">1</span>);</span><br><span class="line">        props.put(<span class="string">&quot;buffer.memory&quot;</span>, <span class="number">33554432</span>);</span><br><span class="line">        props.put(<span class="string">&quot;key.serializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringSerializer&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;value.serializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringSerializer&quot;</span>);</span><br><span class="line"></span><br><span class="line">        Producer producer = <span class="keyword">new</span> KafkaProducer&lt;String, String&gt;(props);</span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">int</span> i = <span class="number">0</span>; i &lt; <span class="number">10</span>; i++) &#123;</span><br><span class="line">            Future future = producer.send(<span class="keyword">new</span> ProducerRecord&lt;String, String&gt;(<span class="string">&quot;first&quot;</span>, <span class="string">&quot;message&quot;</span> + i));</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<p><strong>（2）带回调函数的发送消息</strong></p>
<figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">package com.lytdevproducer;</span><br><span class="line"></span><br><span class="line">import org.apache.kafka.clients.producer.*;</span><br><span class="line"></span><br><span class="line">import java.util.Properties;</span><br><span class="line">import java.util.concurrent.Future;</span><br><span class="line"></span><br><span class="line">public class ProducerWithCallback &#123;</span><br><span class="line">    public static void main(String[] args) &#123;</span><br><span class="line">        Properties props = new Properties();</span><br><span class="line">        props.put(<span class="string">&quot;bootstrap.servers&quot;</span>, <span class="string">&quot;192.168.10.101:9092&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;acks&quot;</span>, <span class="string">&quot;all&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;retries&quot;</span>, 1);</span><br><span class="line">        props.put(<span class="string">&quot;batch.size&quot;</span>, 16384);</span><br><span class="line">        props.put(<span class="string">&quot;linger.ms&quot;</span>, 1);</span><br><span class="line">        props.put(<span class="string">&quot;buffer.memory&quot;</span>, 33554432);</span><br><span class="line">        props.put(<span class="string">&quot;key.serializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringSerializer&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;value.serializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringSerializer&quot;</span>);</span><br><span class="line"></span><br><span class="line">        Producer&lt;String, String&gt; producer = new KafkaProducer&lt;String, String&gt;(props);</span><br><span class="line">        <span class="keyword">for</span> (int i = 0; i &lt; 10; i++) &#123;</span><br><span class="line">            producer.send(new ProducerRecord&lt;String, String&gt;(<span class="string">&quot;first&quot;</span>, <span class="string">&quot;call&quot;</span> + i), new <span class="function"><span class="title">Callback</span></span>() &#123;</span><br><span class="line">                public void onCompletion(RecordMetadata recordMetadata, Exception e) &#123;</span><br><span class="line">                    System.out.println(recordMetadata.toString() + <span class="string">&quot;发送成功，在第&quot;</span> + recordMetadata.partition() + <span class="string">&quot;偏移量&quot;</span> + recordMetadata.offset());</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;);</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h3 id="3-1-3-同步发送API"><a href="#3-1-3-同步发送API" class="headerlink" title="3.1.3 同步发送API"></a>3.1.3 同步发送API</h3><p>kafka消息顺序发送：可以通过发往某一个指定分区+同步发送机制</p>
<figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.chaoyue.dw.kafka.producer;</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.producer.*;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.common.serialization.StringSerializer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.Properties;</span><br><span class="line"><span class="keyword">import</span> java.util.concurrent.ExecutionException;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">MyProducer</span> </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</span><br><span class="line">        Properties props = <span class="keyword">new</span> Properties();</span><br><span class="line">        <span class="comment">//kafka集群，broker-list</span></span><br><span class="line">        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, <span class="string">&quot;192.168.232.202:9092&quot;</span>);</span><br><span class="line">        props.put(ProducerConfig.ACKS_CONFIG, <span class="string">&quot;all&quot;</span>);</span><br><span class="line">        <span class="comment">//重试次数</span></span><br><span class="line">        props.put(ProducerConfig.RETRIES_CONFIG, <span class="number">3</span>);</span><br><span class="line">        <span class="comment">//批次大小</span></span><br><span class="line">        props.put(ProducerConfig.BATCH_SIZE_CONFIG, <span class="number">16384</span>);</span><br><span class="line">        <span class="comment">//等待时间</span></span><br><span class="line">        props.put(ProducerConfig.LINGER_MS_CONFIG, <span class="number">1</span>);</span><br><span class="line">        <span class="comment">//RecordAccumulator缓冲区大小</span></span><br><span class="line">        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, <span class="number">33554432</span>);</span><br><span class="line">        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());</span><br><span class="line">        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());</span><br><span class="line">        KafkaProducer&lt;String, String&gt; kafkaProducer = <span class="keyword">new</span> KafkaProducer&lt;&gt;(props);</span><br><span class="line"></span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">int</span> i = <span class="number">10</span>; i &lt; <span class="number">20</span>; i++) &#123;</span><br><span class="line">            <span class="keyword">try</span> &#123;</span><br><span class="line">                kafkaProducer.send(<span class="keyword">new</span> ProducerRecord&lt;&gt;(<span class="string">&quot;three&quot;</span>,<span class="string">&quot;chaoyue--new-&quot;</span> + i)).get();</span><br><span class="line">            &#125; <span class="keyword">catch</span> (InterruptedException e) &#123;</span><br><span class="line">                e.printStackTrace();</span><br><span class="line">            &#125; <span class="keyword">catch</span> (ExecutionException e) &#123;</span><br><span class="line">                e.printStackTrace();</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line">        kafkaProducer.close();</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h2 id="3-2-Consumer-API"><a href="#3-2-Consumer-API" class="headerlink" title="3.2 Consumer API"></a>3.2 Consumer API</h2><h3 id="3-2-1-自动提交offset"><a href="#3-2-1-自动提交offset" class="headerlink" title="3.2.1 自动提交offset"></a>3.2.1 自动提交offset</h3><figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.lytdevconsumer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.Consumer;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.ConsumerRecord;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.ConsumerRecords;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.KafkaConsumer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.Collections;</span><br><span class="line"><span class="keyword">import</span> java.util.Properties;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">AutoCommitConsumer</span> </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</span><br><span class="line">        Properties props = <span class="keyword">new</span> Properties();</span><br><span class="line">        props.put(<span class="string">&quot;bootstrap.servers&quot;</span>, <span class="string">&quot;192.168.10.101:9092&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;group.id&quot;</span>, <span class="string">&quot;test&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;enable.auto.commit&quot;</span>, <span class="string">&quot;true&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;auto.commit.interval.ms&quot;</span>, <span class="string">&quot;1000&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;key.deserializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringDeserializer&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;value.deserializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringDeserializer&quot;</span>);</span><br><span class="line"></span><br><span class="line">        Consumer&lt;String, String&gt; consumer = <span class="keyword">new</span> KafkaConsumer&lt;String, String&gt;(props);</span><br><span class="line">        consumer.subscribe(Collections.singleton(<span class="string">&quot;first&quot;</span>));</span><br><span class="line">        <span class="keyword">while</span> (<span class="keyword">true</span>)&#123;</span><br><span class="line">            ConsumerRecords&lt;String, String&gt; records = consumer.poll(<span class="number">1000</span>);</span><br><span class="line">            <span class="keyword">for</span> (ConsumerRecord&lt;String, String&gt; record : records)&#123;</span><br><span class="line">                System.out.println(record.value());</span><br><span class="line">            &#125;</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h3 id="3-2-2-手动同步提交offset"><a href="#3-2-2-手动同步提交offset" class="headerlink" title="3.2.2 手动同步提交offset"></a>3.2.2 手动同步提交offset</h3><figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.lytdevconsumer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.Consumer;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.ConsumerRecord;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.ConsumerRecords;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.KafkaConsumer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.Collections;</span><br><span class="line"><span class="keyword">import</span> java.util.Properties;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">SyncCommitConsumer</span> </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</span><br><span class="line">        Properties props = <span class="keyword">new</span> Properties();</span><br><span class="line">        props.put(<span class="string">&quot;bootstrap.servers&quot;</span>, <span class="string">&quot;192.168.10.101:9092&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;group.id&quot;</span>, <span class="string">&quot;test&quot;</span>);</span><br><span class="line"></span><br><span class="line">        props.put(<span class="string">&quot;key.deserializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringDeserializer&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;value.deserializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringDeserializer&quot;</span>);</span><br><span class="line"></span><br><span class="line">        Consumer&lt;String, String&gt; consumer = <span class="keyword">new</span> KafkaConsumer&lt;String, String&gt;(props);</span><br><span class="line">        consumer.subscribe(Collections.singleton(<span class="string">&quot;first&quot;</span>));</span><br><span class="line">        <span class="keyword">while</span> (<span class="keyword">true</span>)&#123;</span><br><span class="line">            ConsumerRecords&lt;String, String&gt; records = consumer.poll(<span class="number">1000</span>);</span><br><span class="line">            <span class="keyword">for</span> (ConsumerRecord&lt;String, String&gt; record : records)&#123;</span><br><span class="line">                System.out.println(record.value());</span><br><span class="line">            &#125;</span><br><span class="line">            consumer.commitSync();</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h3 id="3-2-3-手动异步提交offset"><a href="#3-2-3-手动异步提交offset" class="headerlink" title="3.2.3 手动异步提交offset"></a>3.2.3 手动异步提交offset</h3><figure class="highlight java"><table><tr><td class="code"><pre><span class="line"><span class="keyword">package</span> com.lytdevconsumer;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.clients.consumer.*;</span><br><span class="line"><span class="keyword">import</span> org.apache.kafka.common.TopicPartition;</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> java.util.Collections;</span><br><span class="line"><span class="keyword">import</span> java.util.Map;</span><br><span class="line"><span class="keyword">import</span> java.util.Properties;</span><br><span class="line"></span><br><span class="line"><span class="keyword">public</span> <span class="class"><span class="keyword">class</span> <span class="title">AyncCommitConsumer</span> </span>&#123;</span><br><span class="line">    <span class="function"><span class="keyword">public</span> <span class="keyword">static</span> <span class="keyword">void</span> <span class="title">main</span><span class="params">(String[] args)</span> </span>&#123;</span><br><span class="line">        Properties props = <span class="keyword">new</span> Properties();</span><br><span class="line">        props.put(<span class="string">&quot;bootstrap.servers&quot;</span>, <span class="string">&quot;192.168.10.101:9092&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;group.id&quot;</span>, <span class="string">&quot;test&quot;</span>);</span><br><span class="line"></span><br><span class="line">        props.put(<span class="string">&quot;key.deserializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringDeserializer&quot;</span>);</span><br><span class="line">        props.put(<span class="string">&quot;value.deserializer&quot;</span>, <span class="string">&quot;org.apache.kafka.common.serialization.StringDeserializer&quot;</span>);</span><br><span class="line"></span><br><span class="line">        Consumer&lt;String, String&gt; consumer = <span class="keyword">new</span> KafkaConsumer&lt;String, String&gt;(props);</span><br><span class="line">        consumer.subscribe(Collections.singleton(<span class="string">&quot;first&quot;</span>));</span><br><span class="line">        <span class="keyword">while</span> (<span class="keyword">true</span>)&#123;</span><br><span class="line">            ConsumerRecords&lt;String, String&gt; records = consumer.poll(<span class="number">1000</span>);</span><br><span class="line">            <span class="keyword">for</span> (ConsumerRecord&lt;String, String&gt; record : records)&#123;</span><br><span class="line">                System.out.println(record.value());</span><br><span class="line">            &#125;</span><br><span class="line">            consumer.commitAsync(<span class="keyword">new</span> OffsetCommitCallback() &#123;</span><br><span class="line">                <span class="function"><span class="keyword">public</span> <span class="keyword">void</span> <span class="title">onComplete</span><span class="params">(Map&lt;TopicPartition, OffsetAndMetadata&gt; map, Exception e)</span> </span>&#123;</span><br><span class="line">                    System.out.println(<span class="string">&quot;提交完成&quot;</span>);</span><br><span class="line">                &#125;</span><br><span class="line">            &#125;);</span><br><span class="line">        &#125;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>

<h1 id="四、Flume对接kafka"><a href="#四、Flume对接kafka" class="headerlink" title="四、Flume对接kafka"></a>四、Flume对接kafka</h1><h2 id="4-1-配置flume-flume-kafka-conf"><a href="#4-1-配置flume-flume-kafka-conf" class="headerlink" title="4.1 配置flume(flume-kafka.conf)"></a>4.1 配置flume(flume-kafka.conf)</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># define</span></span><br><span class="line">a1.sources = r1</span><br><span class="line">a1.sinks = k1</span><br><span class="line">a1.channels = c1</span><br><span class="line"></span><br><span class="line"><span class="comment"># source</span></span><br><span class="line">a1.sources.r1.type = <span class="built_in">exec</span></span><br><span class="line">a1.sources.r1.command = tail -F -c +0 /opt/module/flume/data/flume.log</span><br><span class="line">a1.sources.r1.shell = /bin/bash -c</span><br><span class="line"></span><br><span class="line"><span class="comment"># sink</span></span><br><span class="line">a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink</span><br><span class="line">a1.sinks.k1.kafka.bootstrap.servers = hadoop101:9092,hadoop102:9092,hadoop103:9092</span><br><span class="line">a1.sinks.k1.kafka.topic = first</span><br><span class="line">a1.sinks.k1.kafka.flumeBatchSize = 20</span><br><span class="line">a1.sinks.k1.kafka.producer.acks = 1</span><br><span class="line">a1.sinks.k1.kafka.producer.linger.ms = 1</span><br><span class="line"></span><br><span class="line"><span class="comment"># channel</span></span><br><span class="line">a1.channels.c1.type = memory</span><br><span class="line">a1.channels.c1.capacity = 1000</span><br><span class="line">a1.channels.c1.transactionCapacity = 100</span><br><span class="line"></span><br><span class="line"><span class="comment"># bind</span></span><br><span class="line">a1.sources.r1.channels = c1</span><br><span class="line">a1.sinks.k1.channel = c1</span><br></pre></td></tr></table></figure>

<h2 id="4-2-启动flume"><a href="#4-2-启动flume" class="headerlink" title="4.2 启动flume"></a>4.2 启动flume</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/flume-ng agent -c conf/ -n a1 -f <span class="built_in">jobs</span>/flume-kafka.conf</span><br></pre></td></tr></table></figure>

<h2 id="4-3-启动kafka消费者"><a href="#4-3-启动kafka消费者" class="headerlink" title="4.3 启动kafka消费者"></a>4.3 启动kafka消费者</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">bin/kafka-console-consumer.sh --bootstrap-server hadoop101:9092 --from-beginning --topic first</span><br></pre></td></tr></table></figure>

<h2 id="4-4-向flume-log里追加数据"><a href="#4-4-向flume-log里追加数据" class="headerlink" title="4.4 向flume.log里追加数据"></a>4.4 向flume.log里追加数据</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/flume/data</span><br><span class="line">cat &gt;&gt; flume.log</span><br><span class="line">ddddddddd</span><br></pre></td></tr></table></figure>

<h1 id="五、Kafka监控"><a href="#五、Kafka监控" class="headerlink" title="五、Kafka监控"></a>五、Kafka监控</h1><h2 id="5-1-修改kafka配置文件"><a href="#5-1-修改kafka配置文件" class="headerlink" title="5.1 修改kafka配置文件"></a>5.1 修改kafka配置文件</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">vi kafka-server-start.sh</span><br><span class="line"><span class="comment">#修改 KAFKA_HEAP_OPTS</span></span><br><span class="line"><span class="built_in">export</span> KAFKA_HEAP_OPTS=<span class="string">&quot;-server -Xms1g -Xmx1g -XX:PermSize=128m -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:ParallelGCThreads=8 -XX:ConcGCThreads=5 -XX:InitiatingHeapOccupancyPercent=70&quot;</span></span><br><span class="line"><span class="built_in">export</span> JMX_PORT=<span class="string">&quot;9999&quot;</span></span><br></pre></td></tr></table></figure>

<h2 id="5-2-分发配置文件"><a href="#5-2-分发配置文件" class="headerlink" title="5.2 分发配置文件"></a>5.2 分发配置文件</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">xsync kafka-server-start.sh</span><br></pre></td></tr></table></figure>

<h2 id="5-3-Eagle安装"><a href="#5-3-Eagle安装" class="headerlink" title="5.3 Eagle安装"></a>5.3 Eagle安装</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">tar -xzvf kafka-eagle-bin-1.3.7.tar.gz -C /opt/module</span><br><span class="line"><span class="built_in">cd</span> /opt/module</span><br><span class="line">mv kafka-eagle-bin-1.3.7 eagle</span><br></pre></td></tr></table></figure>

<h2 id="5-4-配置环境变量"><a href="#5-4-配置环境变量" class="headerlink" title="5.4 配置环境变量"></a>5.4 配置环境变量</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /etc/profile</span><br><span class="line"><span class="comment"># 添加如下内容</span></span><br><span class="line"><span class="comment">#KE_HOME</span></span><br><span class="line"><span class="built_in">export</span> KE_HOME=/opt/module/eagle</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$PATH</span>:<span class="variable">$KE_HOME</span>/bin</span><br><span class="line"></span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>

<h2 id="5-5-分发配置文件"><a href="#5-5-分发配置文件" class="headerlink" title="5.5 分发配置文件"></a>5.5 分发配置文件</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">xsync /etc/profile</span><br><span class="line"><span class="built_in">source</span> /etc/profile</span><br></pre></td></tr></table></figure>

<h2 id="5-6-修改ke-sh的执行权限"><a href="#5-6-修改ke-sh的执行权限" class="headerlink" title="5.6 修改ke.sh的执行权限"></a>5.6 修改ke.sh的执行权限</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/eagle/bin</span><br><span class="line">chmod a+x ke.sh</span><br></pre></td></tr></table></figure>

<h2 id="5-7-修改system-config-properties"><a href="#5-7-修改system-config-properties" class="headerlink" title="5.7 修改system-config.properties"></a>5.7 修改system-config.properties</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/eagle/conf</span><br><span class="line">vi system-config.properties</span><br><span class="line"><span class="comment"># 修改如下内容</span></span><br><span class="line"><span class="comment"># multi zookeeper&amp;kafka cluster list</span></span><br><span class="line"><span class="comment">######################################</span></span><br><span class="line">kafka.eagle.zk.cluster.alias=cluster1</span><br><span class="line">cluster1.zk.list=hadoop101:2181,hadoop102:2181,hadoop103:2181</span><br><span class="line"></span><br><span class="line"><span class="comment"># kafka offset storage</span></span><br><span class="line"><span class="comment">######################################</span></span><br><span class="line">cluster1.kafka.eagle.offset.storage=kafka</span><br><span class="line"></span><br><span class="line"><span class="comment"># enable kafka metrics</span></span><br><span class="line"><span class="comment">######################################</span></span><br><span class="line">kafka.eagle.metrics.charts=<span class="literal">true</span></span><br><span class="line"></span><br><span class="line"><span class="comment"># kafka jdbc driver address</span></span><br><span class="line"><span class="comment">######################################</span></span><br><span class="line">kafka.eagle.driver=com.mysql.jdbc.Driver</span><br><span class="line">kafka.eagle.url=jdbc:mysql://hadoop101:3306/ke?useUnicode=<span class="literal">true</span>&amp;characterEncoding=UTF-8&amp;zeroDateTimeBehavior=convertToNull</span><br><span class="line">kafka.eagle.username=root</span><br><span class="line">kafka.eagle.password=123456</span><br></pre></td></tr></table></figure>

<h2 id="5-8-重启kafka"><a href="#5-8-重启kafka" class="headerlink" title="5.8 重启kafka"></a>5.8 重启kafka</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="comment"># kafka.sh 为自定义集群启动/停止脚本</span></span><br><span class="line">kafka.sh stop</span><br><span class="line">kafka.sh start</span><br></pre></td></tr></table></figure>

<h2 id="5-9-启动Eagle"><a href="#5-9-启动Eagle" class="headerlink" title="5.9 启动Eagle"></a>5.9 启动Eagle</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line"><span class="built_in">cd</span> /opt/module/eagle/bin</span><br><span class="line">./ke.sh start</span><br></pre></td></tr></table></figure>

<h2 id="5-10-访问Eagle的页面"><a href="#5-10-访问Eagle的页面" class="headerlink" title="5.10 访问Eagle的页面"></a>5.10 访问Eagle的页面</h2><figure class="highlight bash"><table><tr><td class="code"><pre><span class="line">http://192.168.20.101:8048/ke</span><br><span class="line">admin</span><br><span class="line">123456</span><br></pre></td></tr></table></figure>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined" rel="external nofollow noreferrer">liuyuantao</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://liuyuantao.gitee.io/2021/10/12/245dd8355f34.html">https://liuyuantao.gitee.io/2021/10/12/245dd8355f34.html</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" rel="external nofollow noreferrer" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://liuyuantao.gitee.io" target="_blank">全栈进阶那些事</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/%E4%B8%AD%E9%97%B4%E4%BB%B6/">中间件</a><a class="post-meta__tags" href="/tags/Kafka/">Kafka</a></div><div class="post_share"><div class="social-share" data-image="https://img-blog.csdnimg.cn/20200620092054385.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><div class="post-reward"><div class="reward-button button--animated"><i class="fas fa-qrcode"></i> 打赏</div><div class="reward-main"><ul class="reward-all"><li class="reward-item"><a href="/img/qr_code_wechat.jpg" target="_blank"><img class="post-qr-code-img" src="/img/qr_code_wechat.jpg" alt="微信"/></a><div class="post-qr-code-desc">微信</div></li><li class="reward-item"><a href="/img/qr_code_alipay.jpg" target="_blank"><img class="post-qr-code-img" src="/img/qr_code_alipay.jpg" alt="支付宝"/></a><div class="post-qr-code-desc">支付宝</div></li></ul></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/2021/11/12/2d8833eb4290.html"><img class="prev-cover" src="https://img-blog.csdnimg.cn/20200715092910375.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">一文彻底搞懂HBase</div></div></a></div><div class="next-post pull-right"><a href="/2021/09/12/a99eab3d4a13.html"><img class="next-cover" src="https://oscimg.oschina.net/oscnet/20200617165336828.png" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">一文彻底搞懂Flume</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/2021/07/12/1e41bdf3e092.html" title="一文彻底搞懂Zookeeper"><img class="cover" src="https://img-blog.csdnimg.cn/20200518143448459.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2021-07-12</div><div class="title">一文彻底搞懂Zookeeper</div></div></a></div></div></div></div><div class="aside-content" id="aside-content"><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%80%E3%80%81Kafka%E7%9A%84%E7%AE%80%E4%BB%8B"><span class="toc-text">一、Kafka的简介</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#1-1-Kafka%E5%9F%BA%E6%9C%AC%E6%A6%82%E5%BF%B5"><span class="toc-text">1.1 Kafka基本概念</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-2-Kafka%E5%9F%BA%E7%A1%80%E6%9E%B6%E6%9E%84"><span class="toc-text">1.2 Kafka基础架构</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-3-Kafka%E5%B7%A5%E4%BD%9C%E6%B5%81%E7%A8%8B"><span class="toc-text">1.3 Kafka工作流程</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-4-Kafka%E6%96%87%E4%BB%B6%E5%AD%98%E5%82%A8%E6%9C%BA%E5%88%B6"><span class="toc-text">1.4 Kafka文件存储机制</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-5-Kafka%E7%94%9F%E4%BA%A7%E8%80%85"><span class="toc-text">1.5 Kafka生产者</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-5-1-%E5%88%86%E5%8C%BA%E7%AD%96%E7%95%A5"><span class="toc-text">1.5.1 分区策略</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#1-5-2-%E6%95%B0%E6%8D%AE%E5%8F%AF%E9%9D%A0%E6%80%A7%E4%BF%9D%E8%AF%81"><span class="toc-text">1.5.2 数据可靠性保证</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#1-5-3-Exactly-Once%E8%AF%AD%E4%B9%89"><span class="toc-text">1.5.3 Exactly Once语义</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#1-6-Kafka%E6%B6%88%E8%B4%B9%E8%80%85"><span class="toc-text">1.6 Kafka消费者</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#1-6-1-%E6%B6%88%E8%B4%B9%E6%96%B9%E5%BC%8F"><span class="toc-text">1.6.1 消费方式</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#1-6-2-%E5%88%86%E5%8C%BA%E5%88%86%E9%85%8D%E7%AD%96%E7%95%A5"><span class="toc-text">1.6.2 分区分配策略</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#1-6-3-offset%E7%9A%84%E7%BB%B4%E6%8A%A4"><span class="toc-text">1.6.3 offset的维护</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%8C%E3%80%81Kafka%E7%9A%84%E5%AE%89%E8%A3%85%E5%92%8C%E4%BD%BF%E7%94%A8"><span class="toc-text">二、Kafka的安装和使用</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#2-1-Kafka%E5%AE%89%E8%A3%85"><span class="toc-text">2.1 Kafka安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-2-%E4%BF%AE%E6%94%B9%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-text">2.2 修改配置文件</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-3-%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F"><span class="toc-text">2.3 配置环境变量</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-4-%E5%90%AF%E5%8A%A8kafka"><span class="toc-text">2.4 启动kafka</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-5-%E5%81%9C%E6%AD%A2kafka"><span class="toc-text">2.5 停止kafka</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-5-kafka%E5%90%AF%E5%8A%A8%E6%97%A5%E5%BF%97%E6%96%87%E4%BB%B6"><span class="toc-text">2.5 kafka启动日志文件</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-6-%E6%9F%A5%E7%9C%8B%E5%BD%93%E5%89%8D%E6%9C%8D%E5%8A%A1%E5%99%A8%E4%B8%AD%E7%9A%84%E6%89%80%E6%9C%89topic"><span class="toc-text">2.6 查看当前服务器中的所有topic</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-6-%E5%88%9B%E5%BB%BAtopic"><span class="toc-text">2.6 创建topic</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-7-%E6%9F%A5%E7%9C%8Btopic%E8%AF%A6%E6%83%85"><span class="toc-text">2.7 查看topic详情</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-8-%E4%BF%AE%E6%94%B9topic%E5%88%86%E5%8C%BA%E6%95%B0%E9%87%8F"><span class="toc-text">2.8 修改topic分区数量</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-9-%E5%88%A0%E9%99%A4topic"><span class="toc-text">2.9 删除topic</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-10-%E5%8F%91%E9%80%81%E6%B6%88%E6%81%AF"><span class="toc-text">2.10 发送消息</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-11-%E4%BB%8E%E5%A4%B4%E6%B6%88%E8%B4%B9%E6%B6%88%E6%81%AF"><span class="toc-text">2.11 从头消费消息</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-12-%E5%AE%9E%E6%97%B6%E6%B6%88%E8%B4%B9%E6%B6%88%E6%81%AF"><span class="toc-text">2.12 实时消费消息</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-13-%E4%BB%8E%E4%B8%8A%E6%AC%A1%E6%95%85%E9%9A%9C%E6%81%A2%E5%A4%8D%E6%B6%88%E8%B4%B9%E6%B6%88%E6%81%AF"><span class="toc-text">2.13 从上次故障恢复消费消息</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%89%E3%80%81Kafka%E7%9A%84API"><span class="toc-text">三、Kafka的API</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#3-1-Producer-API"><span class="toc-text">3.1 Producer API</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#3-1-1-%E6%B6%88%E6%81%AF%E5%8F%91%E9%80%81%E6%B5%81%E7%A8%8B"><span class="toc-text">3.1.1 消息发送流程</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-1-2-%E5%BC%82%E6%AD%A5%E5%8F%91%E9%80%81API"><span class="toc-text">3.1.2 异步发送API</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-1-3-%E5%90%8C%E6%AD%A5%E5%8F%91%E9%80%81API"><span class="toc-text">3.1.3 同步发送API</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3-2-Consumer-API"><span class="toc-text">3.2 Consumer API</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#3-2-1-%E8%87%AA%E5%8A%A8%E6%8F%90%E4%BA%A4offset"><span class="toc-text">3.2.1 自动提交offset</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-2-2-%E6%89%8B%E5%8A%A8%E5%90%8C%E6%AD%A5%E6%8F%90%E4%BA%A4offset"><span class="toc-text">3.2.2 手动同步提交offset</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#3-2-3-%E6%89%8B%E5%8A%A8%E5%BC%82%E6%AD%A5%E6%8F%90%E4%BA%A4offset"><span class="toc-text">3.2.3 手动异步提交offset</span></a></li></ol></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%9B%9B%E3%80%81Flume%E5%AF%B9%E6%8E%A5kafka"><span class="toc-text">四、Flume对接kafka</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#4-1-%E9%85%8D%E7%BD%AEflume-flume-kafka-conf"><span class="toc-text">4.1 配置flume(flume-kafka.conf)</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-2-%E5%90%AF%E5%8A%A8flume"><span class="toc-text">4.2 启动flume</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-3-%E5%90%AF%E5%8A%A8kafka%E6%B6%88%E8%B4%B9%E8%80%85"><span class="toc-text">4.3 启动kafka消费者</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-4-%E5%90%91flume-log%E9%87%8C%E8%BF%BD%E5%8A%A0%E6%95%B0%E6%8D%AE"><span class="toc-text">4.4 向flume.log里追加数据</span></a></li></ol></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%94%E3%80%81Kafka%E7%9B%91%E6%8E%A7"><span class="toc-text">五、Kafka监控</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#5-1-%E4%BF%AE%E6%94%B9kafka%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-text">5.1 修改kafka配置文件</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-2-%E5%88%86%E5%8F%91%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-text">5.2 分发配置文件</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-3-Eagle%E5%AE%89%E8%A3%85"><span class="toc-text">5.3 Eagle安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-4-%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F"><span class="toc-text">5.4 配置环境变量</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-5-%E5%88%86%E5%8F%91%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6"><span class="toc-text">5.5 分发配置文件</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-6-%E4%BF%AE%E6%94%B9ke-sh%E7%9A%84%E6%89%A7%E8%A1%8C%E6%9D%83%E9%99%90"><span class="toc-text">5.6 修改ke.sh的执行权限</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-7-%E4%BF%AE%E6%94%B9system-config-properties"><span class="toc-text">5.7 修改system-config.properties</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-8-%E9%87%8D%E5%90%AFkafka"><span class="toc-text">5.8 重启kafka</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-9-%E5%90%AF%E5%8A%A8Eagle"><span class="toc-text">5.9 启动Eagle</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-10-%E8%AE%BF%E9%97%AEEagle%E7%9A%84%E9%A1%B5%E9%9D%A2"><span class="toc-text">5.10 访问Eagle的页面</span></a></li></ol></li></ol></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By liuyuantao</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">扫描二维码,关注我们!了解更多全栈技能!<br/><img style="width:100px;height:100px;" src="/img/qrcode_for_gh_da91cebd00cc_258.jpg" ></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="font-plus" type="button" title="放大字体"><i class="fas fa-plus"></i></button><button id="font-minus" type="button" title="缩小字体"><i class="fas fa-minus"></i></button><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instant.page/instantpage.min.js" type="module"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"></div><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>